r"""
 MARCImport.py

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.

 Copyright 2010- Jeremy Nelson, Tomichi Informatics
"""
import re,datetime,sys
import MODSData,basedata,FRBRData
from google.appengine.ext import db
from google.appengine.api import users
import logging
sys.path.insert(0,'common/pymarc.zip')
from pymarc import MARCReader


class pymarcToFRBR():
    r""" class wraps pymarc libraries for parsing of MARC files into
    eCataloger 1.2 FRBR entities.
    """

    def __init__(self,
                 **args):
        """ constructor allows for either file name or raw MARC binary
        record to be parsed. Constructor also creates empty FRBR
        Work-Expression-Manifestation-Item entities for later use
        min the parser.

        args:
        file -- optional, file and path to MARC binary record
        raw_marc -- optional, raw MARC binary
        work -- optional, FRBR Work instance
        expression -- optional, FRBR Expression instance
        manifestation -- optional, FRBR Manifestation instance
        item -- optional, FRBR Item instance
        record -- optional, MODS recordInfo instance
        endeavor -- optional, FRBR Endeavor instance
        """
        if args.has_key('file'):
            reader_src = open(args['file'])
        elif args.has_key('raw_marc'):
            reader_src = args['raw_marc']
        else:
            reader_src = None
        if reader_src:
            self.marc_reader = MARCReader(reader_src) 
        else:
           self.marc_reader = None
        if args.has_key('work'):
            self.work = args['work']
        else:
            self.work = FRBRData.Work(created_by=users.get_current_user())
        if args.has_key('expression'):
            self.expression = args['expression']
        else:
            self.expression = FRBRData.Expression(created_by=users.get_current_user())
        if args.has_key('manifestation'):
            self.manifestation= args['manifestation']
        else:
            self.manifestation = FRBRData.Manifestation(created_by=users.get_current_user())
        if args.has_key('item'):
            self.item = args['item']
        else:
            self.item = FRBRData.Item(created_by=users.get_current_user())
        if args.has_key('record'):
            self.record = args['record']
        else:
            self.record = MODSData.recordInfo(created_by=users.get_current_user())
        if args.has_key('endeavor'):
            self.endeavor = args['endeavor']
        else:
            self.endeavor = FRBRData.Endeavor(created_by=users.get_current_user())

    def search_id(self,type_of,value,is_valid=True,status=None):
        """ method searches and returns MODSData.identifier

        args:
        type_of -- required, type of identifier (issn,isbn,etc.)
        value -- required, value of identifier
        is_valid -- optional, boolean if identifier is valid
        status -- optional, values should be "incorrect" or "cancelled"
        """
        id_query = db.GqlQuery('''SELECT __key__ FROM identifier
                                   WHERE type=:1 AND value=:2
                                   AND is_valid=:3 AND status=:4''',
                               type_of,
                               value.strip(),
                               is_valid,
                               status)
        id_results = id_query.fetch(10)
        if len(id_results) == 1:
            return id_results[0]
        else:
            return id_results

    def search_note(self,type_of,raw_note):
        r""" method searches for and returns MODSData.note if found

        args:
        type_of -- required, type of note (i.e. description, etc.)
        raw_note -- required, raw string value of note
        """
        note_query = db.GqlQuery('''SELECT __key__ FROM note
                                    WHERE type=:1 AND value=:2''',
                                 type_of,
                                 raw_note.strip())
        note_results = note_query.get()
        if note_results:
            return note_results
        else:
            return None

    def classification_exists_or_add(self,
                                     authority,
                                     call_number,
                                     item_number=None):
        r""" method returns existing MODSData.classification if found
        or created and returns new MODSData.classification

        args:
        authority -- required, must be value in MODSData.classification.authority
        call_number -- required, classification number
        item_number -- optional, stores item number
        """
        query = MODSData.classification.all().filter('authority =',authority)
        query.filter('call_number =',call_number)
        if item_number:
            query.filter('item_number =',item_number)
        classification = query.get()
        if not classification:
            classification = MODSData.classification(\
                created_by=users.get_current_user(),
                authority=authority,
                call_number=call_number)
            if item_number:
                classification.item_number = item_number
            classification.put()
        return classification
        
                                     

    def id_exists_or_add(self,
                         type_of,
                         raw_value,
                         is_valid=True,
                         status=None):
        r""" methods returns existing MODSData.identifier if found or
        creates and returns new MODSData.identifier

        args:
        type_of -- required, type of identifier, (i.e. issn,isbn)
        raw_value -- required, raw string value of identifier
        is_valid -- optional, boolean if identifier is valid
        status -- optional, should be restricted to "incorrect" or "cancelled"
        """
        id_exists = self.search_id(type_of,raw_value,is_valid,status)
        if id_exists:
            if type(id_exists) == list:
                for key in id_exists:
                    if not self.manifestation.identifiers.count(str(key)):
                        self.manifestation.identifiers.append(key)
            else:
                if not self.manifestation.identifiers.count(str(id_exists)):
                    self.manifestation.identifiers.append(id_exists)
        else:
            new_identifier = MODSData.identifier(type=type_of,
                                                 value=raw_value.strip(),
                                                 is_valid=bool(is_valid),
                                                 display_form="%s %s" %\
                                                 (raw_value.strip(),
                                                  type_of.upper()))
            if status:
                new_identifier.status = status
            if users.get_current_user():
                new_identifier.created_by = users.get_current_user()
            new_identifier.put()
            self.manifestation.identifiers.append(new_identifier.key())
        self.manifestation.put()
        if id_exists:
            return id_exists
        elif new_identifier:
            return new_identifier.key()
        else:
            return None

    def process_leader(self,leader):
        r""" method parses and recasts eCataloger entities from MARC record's
        leader.
        <http://www.loc.gov/marc/bibliographic/bdleader.html>

        args:
        leader -- required, raw string of leader
        """
        type_of = {"a":"language material",
                   "c":"notated music",
                   "d":"manuscript notated music",
                   "e":"cartographic material",
                   "f":"manuscript cartographic material",
                   "g":"projected medium",
                   "i":"nonmusical sound recording",
                   "j":"musical sound recording",
                   "k":"two-dimensional nonprojectable graphic",
                   "m":"computer file",
                   "o":"kit",
                   "p":"mixed materials",
                   "r":"three-dimensional artifact or naturally occurring object",
                   "t":"manuscript language material"}
        bibliographic_level = {"a":"monographic component part",
                               "b":"serial component part",
                               "c":"collection",
                               "d":"subunit",
                               "i":"integrating resource",
                               "m":"monograph/item",
                               "s":"serial"}
        rec_type = leader[6]
        bibl_level = leader[7]
        # Follows descriptive rules to recast eCataloger FRBR entities
        # using leader dependencies rules
        # Book
        if rec_type == 'a' and (bibl_level =='a' or bibl_level =='m'):
            printedBook = FRBRData.PrintedBook()
            self.manifestation = printedBook.recast(self.manifestation)
        # Manuscript
        elif rec_type == 't':
            self.expression.is_manuscript = True
            work = FRBRData.PublicationWork()
            self.work = work.recast(self.work)
            manuscript = FRBRData.Manuscript()
            self.manifestation = manuscript.recast(self.manifestation)
            endeavor = FRBRData.PrintedBookEndeavor()
            self.endeavor = endeavor.recast(self.endeavor)         
        # Magazine, Journal, or Newspaper
        elif rec_type == 'a' and ['b','i','s'].count(bibl_level) > 0:
            serial_work = FRBRData.SerialWork()
            self.work = serial_work.recast(self.work)
            serial = FRBRData.Serial(frequency="completely irregular")
            self.expression = serial.recast(self.expression)
            periodical = FRBRData.Periodical(publication_status="current")
            self.manifestation = periodical.recast(self.manifestation)
        # Musical work
        elif ['c','d','i','j'].count(rec_type) > 0:
            if ['c','d'].count(rec_type) > 0:
                music_work = FRBRData.MusicalWork()
                self.work = music_work.recast(self.work)
                notated_music = FRBRData.NotatedMusic()
                self.manifestation = notated_music.recast(self.manifestation)
            elif rec_type == 'd':
                self.expression.is_manuscript = True
            elif ['i','j'].count(rec_type) > 0:
                recording_work = FRBRData.RecordingWork()
                self.work = recording_work.recast(self.work)
                recording = FRBRData.Recording()
                self.expression = recording.recast(self.expression)
                sound_recording = FRBRData.SoundRecording()
                self.manifestation = sound_recording.recast(self.manifestation)
                if rec_type == 'i':
                    self.expression.form = 'sound recording-nonmusical'
                elif rec_type == 'j':
                    self.expression.form = 'sound recording-musical'
        # Maps
        elif ['e','f'].count(rec_type) > 0:
            map_work = FRBRData.CartographicWork(form="map")
            self.work = map_work.recast(self.work)
            map_manifestation = FRBRData.CartographicResource()
            self.manifestation = map_manifestation.recast(self.manifestation)
            if rec_type == 'e':
                cartographic_image_obj = FRBRData.CartographicImageObject()
                self.expression = cartographic_image_obj.recast(self.expression)
                map_endeavor = FRBRData.MapEndeavor()
                self.endeavor = map_endeavor.recast(self.endeavor)
            if rec_type == 'f':
                self.expression.is_manuscript = True
        # Visual Materials
        elif ['g','k','o','r'].count(rec_type) > 0:
            if rec_type == 'g':
                projected_image = FRBRData.ProjectedImage()
                self.expression = projected_image.recast(self.expression)
                visual_projection = FRBRData.VisualProjection()
                self.manifestation = visual_projection.recast(self.manifestation)
            elif rec_type == 'k':
                pass
            # Kit
            elif rec_type == 'o':
                new_genre = MODSData.genre(created_by=users.get_current_user(),
                                           authority="marcgt",
                                           value="kit")
                new_genre.put()
                self.expression.genre.append(new_genre.key())
            elif rec_type == 'r':
                pass
        # Computer Files
        elif rec_type == 'm':
            pass
        # Mixed Materials
        elif rec_type == 'p':
            pass
        self.work.put()
        self.expression.put()
        self.manifestation.put()
        self.endeavor.put()

    def book_008(self,fixed_values):
        r""" method takes the fixed value string from the 008 MARC field
        and updates FRBR and MODS entities based on values found at
        <http://www.loc.gov/marc/bibliographic/bd008b.html>

        args:
        fixed_values -- required, string of fixed values from 008 MARC field
        """
        # Controlled MARC vocabulary 
        illustrations = {"a":"illustrations",
                         "b":"maps",
                         "c":"portraits",
                         "d":"charts",
                         "e":"plans",
                         "f":"plates",
                         "g":"music",
                         "h":"facsimiles",
                         "i":"coats of arms",
                         "j":"genealogical tables",
                         "k":"forms",
                         "l":"samples",
                         "m":"phonodisc, phonowire, etc.",
                         "o":"photographs",
                         "p":"illuminations"}
        audiences = {"a":"preschool",
                     "b":"primary",
                     "c":"pre-adolescent",
                     "d":"adolescent",
                     "e":"adult",
                     "f":"specialized",
                     "g":"general",
                     "j":"juvenile"}
        form_of_item = {"a":"microfilm",
                        "b":"microfiche",
                        "c":"microopaque",
                        "d":"large print",
                        "f":"braille",
                        "o":"online",
                        "q":"direct electronic",
                        "r":"regular print reproduction",
                        "s":"electronic"}
        nature_contents = {"a":"abstract or summary",
                           "b":"bibliography",
                           "c":"catalog",
                           "d":"dictionary",
                           "e":"encyclopedia",
                           "f":"handbook",
                           "g":"legal article",
                           "i":"index",
                           "j":"patent",
                           "k":"discography",
                           "l":"legislation",
                           "m":"thesis",
                           "n":"survey of literature",
                           "o":"review",
                           "p":"programmed text",
                           "q":"filmography",
                           "r":"directory",
                           "s":"statistics",
                           "t":"technical report",
                           "u":"standards or specifications",
                           "v":"legal case and case notes",
                           "w":"law report and digest",
                           "y":"yearbook",
                           "z":"treatie",
                           "2":"offprint",
                           "5":"calendar",
                           "6":"comic of graphic novel"}
        gov_publications = {"a":"autonomous or semi-autonomous",
                            "c":"multilocal",
                            "f":"federal/national",
                            "i":"international intergovernmental",
                            "l":"local",
                            "m":"multistate",
                            "o":"government publication-level undetermined",
                            "s":"state, etc.",
                            "u":"unknown",
                            "z":"other"}
        literary_form = {"0":"not fiction (general)",
                        "1":"fiction",
                        "d":"drama",
                        "e":"essay",
                        "f":"novel",
                        "h":"humor, satire",
                        "i":"letter",
                        "j":"short story",
                        "m":"mixed forms",
                        "p":"poetry",
                        "s":"speech",
                        "u":"unknown"}
        biography = {"a":"autobiography",
                     "b":"biography",
                     "c":"collective biography",
                     "d":"contains biographical information"}
        def add_marc_genre(genre_value):
            new_genre = MODSData.genre.get_or_insert(\
                    authority="marcgt",
                    value=genre_value)
            if self.expression.genre.count(new_genre.key()) < 1:
                self.expression.genre.append(new_genre.key())
        # Creates new illustrative content and associates with expression
        for value in fixed_values[18:22]:
            if illustrations.has_key(value):
                new_content = FRBRData.IllustrativeContent.get_or_insert(\
                    'IllustrativeContent',
                    created_by=users.get_current_user(),
                    type_of=illustrations[value])
                self.expression.illustrative_content.append(new_content.key())
        # Checks and sets audience for FRBRData.Work
        if audiences.has_key(fixed_values[22]):
            self.work.intended_audiences.append(audiences[fixed_values[23]])
        # Checks and sets form, depending on value, can change FRBR and MODS
        # entities
        if form_of_item.has_key(fixed_values[23]):
            item_form_code = fixed_values[23]
            if ['a','b','c'].count(item_form_code) > 0:
                if item_form_code == 'a':
                    # Default value, should change depending on
                    # specific type of microfilm see FRBRData.Microform.carrier
                    carrier = 'microfilm roll'
                else:
                    carrier = form_of_item[item_form_code]
                new_microform = FRBRData.Microform(\
                    created_by=users.get_current_user(),
                    carrier=carrier)
                self.manifestation = new_microform.recast(self.manifestation)
            elif item_form_code == 'd':
                self.work.form = 'large print'
                self.manifestation.rda_font_size='large print'
            elif item_form_code == 'f':
                self.work.form = 'braille'
                self.expression.tactile_notation = 'braille code'
            elif ['o','q','s'].count(item_form_code) > 0:
                if item_form_code == 'o':
                    new_resource = FRBRData.RemoteAccessElectronicResource(\
                        created_by=users.get_current_user(),
                        mode_of_access="internet")
                    self.manifestation = new_resource.recast(self.manifestation)
                elif item_form_code == 'q':
                    new_resource = FRBRData.ElectronicResource(\
                        created_by=users.get_current_user())
                    self.manifestation = new_resource.recast(self.manifestation)
        # Nature of Content, variable processing depending on value
        for value in fixed_values[24:28]:
            if nature_contents.has_key(value):
                add_marc_genre(nature_contents[value])
        # Government publication
        if gov_publications.has_key(fixed_values[28]):
            gov_code = fixed_values[28]
            if gov_code == 'f':
                fed_doc = FRBRData.FederalDocument()
                self.manifestation = fed_doc.recast(self.manifestation)
        # Conference Publications
        if fixed_values[29] == '1':
            self.event = FRBRData.Event(created_by=users.get_current_user(),
                                        type="conference")
            self.event.put()
            self.work.subjects.append(self.event.key())
            add_marc_genre("conference publication")
        # Festschrift
        if fixed_values[30] == '1':
            add_marc_genre("festschrift")
        # Index, first checks value in self.expression.genre before
        # adding
        if fixed_values[31] == '1':
            add_marc_genre("index")
        # Literary form
        if literary_form.has_key(fixed_values[33]):
            add_marc_genre(literary_form[fixed_values[33]])
        # Biography
        if biography.has_key(fixed_values[34]):
            add_marc_genre(biography[fixed_values[34]])
        self.work.put()
        self.expression.put()
        self.manifestation.put()
        self.endeavor.put()

    def music_008(self,fixed_values):
        r""" takes the fixed value string from the 008 MARC field
        and updates FRBR and MODS entities based on values found at
        <http://www.loc.gov/marc/bibliographic/bd008m.html> for Music

        args:
        fixed_values -- required,  string of fixed values from 008 MARC field
        """
        compostion_forms = {"an":"anthems",
                            "bd":"ballads",
                            "bg":"bluegrass music",
                            "bl":"blues",
                            "bt":"ballets",
                            "ca":"chaconnes",
                            "cb":"chants, other religions",
                            "cc":"chant, christian",
                            "cg":"concerti grossi",
                            "ch":"chorales",
                            "cl":"chorale preludes",
                            "cn":"canons and rounds",
                            "co":"concertos",
                            "cp":"chansons, polyphonic",
                            "cr":"carols",
                            "cs":"chance compositions",
                            "ct":"cantatas",
                            "cy":"country music",
                            "cz":"canzonas",
                            "df":"dance forms",
                            "dv":"divertimentos, serenades, cassations, divertissements, and notturni",
                            "fg":"fugues",
                            "fl":"flamenco",
                            "fm":"folk music",
                            "ft":"fantasias",
                            "gm":"gospel music",
                            "hy":"hymns",
                            "jz":"jazz",
                            "mc":"musical revues and comedies",
                            "md":"madrigals",
                            "mi":"minuets",
                            "mo":"motets",
                            "mp":"motion picture music",
                            "mr":"marches",
                            "ms":"masses",
                            "mu":"multiple forms",
                            "mz":"mazurkas",
                            "nc":"nocturnes",
                            "nn":"not applicable",
                            "op":"operas",
                            "or":"oratorios",
                            "ov":"overtures",
                            "pg":"program music",
                            "pm":"passion music",
                            "po":"polonaises",
                            "pp":"popular music",
                            "pr":"preludes",
                            "ps":"passacaglias",
                            "pt":"part-songs",
                            "pv":"pavans",
                            "rc":"rock music",
                            "rd":"rondos",
                            "rg":"ragtime music",
                            "ri":"ricercars",
                            "rp":"rhapsodies",
                            "rq":"requiems",
                            "sd":"square dance music",
                            "sg":"songs",
                            "sn":"sonatas",
                            "sp":"symphonic poems",
                            "st":"studies and exercises",
                            "su":"suites",
                            "sy":"symphonies",
                            "tc":"toccatas",
                            "tl":"teatro lirico",
                            "ts":"trio-sonatas",
                            "uu":"unknown",
                            "vi":"villancicos",
                            "vr":"variations",
                            "wz":"waltzes",
                            "za":"zarzuelas",
                            "zz":"other"}
        format_music = {"a":"full score",
                        "b":"full score, miniature or study size",
                        "c":"accompaniment reduced for keyboard",
                        "d":"voice score",
                        "e":"condensed score or piano-conductor score",
                        "g":"close score",
                        "h":"chorus score",
                        "i":"condensed score",
                        "j":"performer-conductor part",
                        "m":"multiple score formats",
                        "n":"not applicable",
                        "u":"unknown",
                        "z":"other"}
        # Form of composition
        for form in fixed_values[18:20]:
            if compostion_forms.has_key(form):
                if self.work.composition_forms.count(form) < 1:
                    self.work.composition_forms.append(\
                        compostion_forms[form])
        # Format of Music
        if format_music.has_key(fixed_values[20]):
            format_code = fixed_values[20]
            if hasattr(self.expression,"type_of"):
                self.expression.type_of=format_code
        self.work.put()
        self.expression.put()
        
            
        
        
                    
            

    def process_fixed_length(self,marc_field):
        r""" method parses and updates eCataloger entities from
         fixed length 008 MARC field

         args:
         marc_field -- required, 008 MARC field
         """
        fixed_values = marc_field.value()
        if type(self.manifestation) == FRBRData.PrintedBook or\
           type(self.manifestation) == FRBRData.Manuscript:
            self.book_008(fixed_values)
        elif type(self.work) == FRBRData.MusicWork or\
             type(self.work) == FRBRData.RecordingWork:
            self.music_008(fixed_values)
        

    def add_lccn(self,marc_field):
        r""" method parses and updates eCataloger entities from
        010 MARC field 

        args:
        marc_field -- required, 010 MARC raw field from the Library of Congress
        """
        if marc_field['a']:
            raw_lccn = marc_field['a']
            self.id_exists_or_add('lccn',raw_lccn)
        if marc_field['b']:
            raw_nucmc = marc_field['b']
            self.id_exists_or_add('nucmc',raw_nucmc)
        if marc_field['z']:
            raw_value = marc_field['z']
            invalid_exists = self.search_id(["lccn","nucmc"],
                                            raw_value.strip())
            if invalid_exists:
                for value in invalid_exists:
                    invalid_id = db.get(value)
                    if self.manifestation.identifiers.count(str(value)):
                        invalid_id.is_valid = False
                        invalid_id.put()
                    else:
                        invalid_id.is_valid = False
                        invalid_id.put()
                        self.manifestation.identifiers.append(str(value))
            else:
                invalid_lccn = MODSData.identifier(type='lccn',
                                                   value=raw_value,
                                                   is_valid=False,
                                                   display_form=\
                                                   "%s Invalid LCCN" % raw_value)
                invalid_lccn.put()
                self.manifestation.identifiers.append(invalid_lccn.key())
        self.manifestation.put()

    def add_isbn(self,marc_fields):
        r""" method parses ISBN 020 MARC field ISBN Number to eCataloger
        entities and properties.

        args:
        marc_fields -- required, list of 020 MARC fields
        """
        isbn_re = re.compile(r'([0-9\-]+)')
        pbk_re = re.compile(r'\(pbk.\)')
        counter = 0
        for isbn_field in marc_fields:
            isbn_raw = isbn_field.get_subfields('a')[0]
            isbn_field = isbn_re.search(isbn_raw).groups()[0].replace("-",'')
            isbn_key = self.id_exists_or_add('isbn',isbn_field)
            # Assumes first 020 field should be set to Manifestation's ISBN
            # property
            if hasattr(self.manifestation,"isbn"):                
                if not self.manifestation.isbn and counter is 0:
                    self.manifestation.isbn = db.get(isbn_key)
            if pbk_re.search(isbn_raw):
                pass
            counter += 1
        self.manifestation.put()
                
        

    def add_issn(self,marc_field):
        r""" method parses ISSN 022 MARC field ISSN Number and then adds
        to eCataloger entities and properties.

        args:
        marc_field -- required, 022 MARC field
        """
        if type(self.expression) != FRBRData.Serial:
            serial = FRBRData.Serial(frequency="completely irregular")
            self.expression = serial.recast(self.expression)
        if type(self.work) != FRBRData.SerialWork:
            serial_work = FRBRData.SerialWork()
            self.work = serial_work.recast(self.work)
        raw_issn = marc_field.get_subfields('a')[0]
        if raw_issn:
            id_key = self.id_exists_or_add('issn',raw_issn)
            self.expression.issn = db.get(id_key)
        raw_issnl = marc_field.get_subfields('l')[0]
        if raw_issnl:
            id_key = self.id_exists_or_add('issnl',raw_issnl)
            self.expression.issnl = db.get(id_key)
        self.expression.put()
        self.manifestation.put()

    def add_system_ctrl_num(self,marc_field):
        r""" method parses 035 System Control Number and creates
         MODSData.recordIdentifier entities and associates with
         MODSData.recordInfo

         args:
         marc_field -- required, 035 MARC field
         """
        ctrl_fld_re = re.compile(r"\((\w+)\)(\w+)")
        def add_record_id(info,is_valid):
            org_code,value = ctrl_fld_re.search(info).groups()
            query = db.GqlQuery('''SELECT * FROM CorporateBody
                                          WHERE codes=:1''',
                                         org_code)
            existing_org = query.get()
            if not existing_org:
                existing_org = FRBRData.CorporateBody(\
                    codes=[str(org_code),],
                    identification_status='preliminary')
                existing_org.put()
            rec_id = MODSData.recordIdentifier(\
                authority=existing_org,
                value=value,
                is_valid=is_valid)
            return rec_id.put()
        if marc_field['a']:
            subfields = marc_field.get_subfields('a')
            for a_field in subfields:
                self.record.record_identifiers.append(\
                    add_record_id(a_field,True))
        if marc_field['z']:
            subfields = marc_field.get_subfields('z')
            for z_field in subfields:
                self.record.record_identifiers.append(\
                    add_record_id(z_field,False))
        self.record.put()


    def add_catalog(self,marc_field):
        r"""  method parses 040 Cataloging source and updates or adds
        eCataloger organizations.

        args:
        marc_field -- required, 040 MARC field
        """
        tmp_corp_dict = {}
        def check_or_add_agency(code_list):
            for code in code_list:
                if not tmp_corp_dict.has_key(code):
                    query = db.GqlQuery('''SELECT __key__ FROM CorporateBody
                                          WHERE codes=:1''',
                                         code)
                    existing_corp_key = query.get()
                    if existing_corp_key:
                        tmp_corp_dict[code] = db.get(existing_corp_key)
                    else:
                        new_corp = FRBRData.CorporateBody(\
                            codes=[code,],
                            identification_status='preliminary')
                        if users.get_current_user():
                            new_corp.created_by = users.get_current_user()
                        new_corp.put()
                        tmp_corp_dict[code] = new_corp
        org_agency_codes = marc_field.get_subfields('a')
        check_or_add_agency(org_agency_codes)
        self.record.org_cataloging_agency = tmp_corp_dict[org_agency_codes[0]]
        transcribing_agency_codes = marc_field.get_subfields('c')
        check_or_add_agency(transcribing_agency_codes)
        self.record.transcribing_agency = tmp_corp_dict[org_agency_codes[0]]
        modifiying_agency_codes = marc_field.get_subfields('d')
        check_or_add_agency(modifiying_agency_codes)
        for code in modifiying_agency_codes:
            if not self.record.modifying_agencies.count(tmp_corp_dict[code].key()):
                self.record.modifying_agencies.append(tmp_corp_dict[code].key())
        self.record.put()

    def add_lc_call_number(self,marc_field):
        r""" method parses 050 Library of Congress Call Number into eCataloger
        MODS entities and associated FRBR entities

        args:
        marc_field -- required, 050 MARC field
        """
        if marc_field.indicators:
            if marc_field.indicators[0]:
                pass
            if marc_field.indicators[1]:
                pass
        raw_class_num = marc_field.get_subfields('a')[0]
        raw_item_number = marc_field.get_subfields('b')[0]
        classification_entity = self.classification_exists_or_add(\
            authority="lcc",
            call_number=raw_class_num,
            item_number=raw_item_number)
        if not self.manifestation.classifications.count(classification_entity.key()) >\
           0:
            self.manifestation.classifications.append(classification_entity.key())
        self.manifestation.put()

    def add_dewey_call_number(self,marc_field):
        r""" method parses 082 Dewey Decimal Classification Number (R) MARC
         field into eCataloger MODS entities and associated FRBR entities

         args:
         marc_field -- required, 082 MARC field
         """
        classification_numbers = marc_field.get_subfields('a')
        item_number,designation,edition = None,None,None
        if marc_field.get_subfields('b'):
            item_number = marc_field.get_subfields('b')[0]
        if marc_field.get_subfields('m'):
            value = marc_field.get_subfields('m')[0]
            if value == 'a':
                designation = "standard"
            elif value == 'b':
                designation = "optional"
        if marc_field.get_subfields('2'):
            edition = marc_field.get_subfields('2')[0]
        for number in classification_numbers:
            classification_entity = self.classification_exists_or_add(\
                authority="ddc",
                call_number=number,
                item_number=item_number)
            if designation:
                classification_entity.designation = designation
            if edition:
                classification_entity.edition = edition
            if not self.manifestation.classifications.count(\
                classification_entity.key()) > 0:
                self.manifestation.classifications.append(\
                classification_entity.key())
            classification_entity.put()
        self.manifestation.put()

    def __check_role__(self,role_code):
        """
        Internal method takes a role code, checks datastore
        for role code or adds to datastore

        Parameters:
        `role_code`: Required, LOC MODS role code
        """
        role_query = db.GqlQuery("SELECT * FROM role WHERE code=:1",role_code)
        role = role_query.get()
        if not role:
            role = MODSData.role(code=role_code)
            role.put()
        return role

    def __check_add_author__(self,entity_key):
        """
        Internal method Checks to see if entity is already associated with the work as
        an author, if not adds author to work

        Parameters:
        `entity_key`: Entity key to check
        """
        author_role = self.__check_role__('aut')
        work_creators = self.work.get_creators()
        if work_creators:
            for entity in work_creators:
                if entity.creator.key() == entity_key:
                    if entity.get_roles().count(author_role.key()) < 0:
                        entity.roles.append(author_role.key())
        else:
            new_author = FRBRData.Creators(creator=db.get(entity_key),
                                           roles=[])
            role_key_str = str(author_role.key())
            new_author.roles.append(db.Key(role_key_str))
            new_author.put()
            self.work.creators.append(new_author.key())
        self.work.put()



    def add_personal_name(self,marc_field):
        r""" method parses X00 Personal Name MARC fields into eCataloger
         FRBR entities.

         args:
         marc_field -- required, X00 MARC field
         """
        first_name,last_name,middle = None,None,[]
        subfield_a = marc_field.get_subfields('a')[0]
        if marc_field.indicators:
            if marc_field.indicators[0] == '0':
                display_form = subfield_a
                # Ad Hoc generation of a last name from the last
                # word when the subfield is split by words
                last_name = subfield_a.split(" ")[-1]
            # The logic here roughly follows <surname, firstname middle>
            # format
            elif marc_field.indicators[0] == '1':
                last_name,raw_remaining = subfield_a.split(",")
                rest_of = raw_remaining.lstrip().split(" ")
                first_name = rest_of[0].replace(".","")
                for name in rest_of[1:]:
                    middle.append(name.strip().replace(".",""))
            elif marc_field.indicators[0] == '3':
                last_name = sub_field_a.strip()
        numeration = marc_field.get_subfields('b')
        titles = marc_field.get_subfields('c')
        raw_dates = marc_field.get_subfields('d')
        relator_terms = marc_field.get_subfields('e')
        # Search for existing FRBR Person entity
        query = db.GqlQuery("SELECT __key__ FROM Person WHERE family=:1 AND given=:2",
                            last_name,first_name)
        results = query.fetch(1000)
        if not results:
            new_person = FRBRData.Person(family=last_name)
            if first_name:
                new_person.given = first_name
            if len(middle) > 0:
                for name in middle:
                    new_person.middle += ' %s' % name
            if len(titles) > 0:
                for title in titles:
                    new_person.terms_of_address.append(title)
            if len(raw_dates) > 0:
                for raw_date in raw_dates:
                    date_range = raw_date.split("-")
                    if date_range[0]:
                        new_person.birth= basedata.BaseDate(\
                            year=int(date_range[0]))
                    if date_range[1]:
                        raw_death = date_range[1]
                        if not re.search(r'\d',raw_death[-1]):
                            raw_death = raw_death[:-1]
                        new_person.death = basedata.BaseDate(\
                            year=int(raw_death))
            if len(relator_terms) > 0:
                pass
            person_key = new_person.put()
        else:
            if len(results) == 1:
                person_key = results[0].key()
            #! Need to work on testing middle name, and other values in
            #! other MARC 100 subfields
            else:
                pass
        self.__check_add_author__(person_key)

    def add_corporate_creators(self,marc_field):
        """
        Method checks/adds Work creator that is a CorporateBody, 
        processes MARC 110 field.

        Parameters:
        `marc_field`: Required, MARC 110 field
        """
        end_punc_re = re.compile(r'[:.]')
        def check_or_add_corporateBody(raw_name):
            if end_punc_re.search(raw_name[-1]):
                raw_name = raw_name[:-1]
            query = db.GqlQuery('SELECT __key__ FROM CorporateBody WHERE name=:1',
                                raw_name.lower())
            result = query.get()
            if not result:
                corporate_creator = FRBRData.CorporateBody(name=raw_name.lower(),
                                                           display_form=raw_name,
                                                           identification_status="preliminary")
            else:
                corporate_creator = db.get(result)
            return corporate_creator
        if marc_field.tag != '110':
            raise ValueError("add_corporate_creators requres a MARC 110 field,"\
                             " marc_field is %s" % marc_field.tag)
        first_indicator = marc_field.indicators[0]
        subfields_a = marc_field.get_subfields('a')
        subfields_b = marc_field.get_subfields('b')
        # Basic form, check/create CorporateBody with subfield a
        if first_indicator == '0':
            raw_name = subfields_a[0]
            corporate_creator = check_or_add_corporateBody(raw_name)
        elif first_indicator == '1':
            raw_jurisdiction = subfields_a[0]
            if not re.search(r'\w+',raw_jurisdiction[-1]):
                raw_jurisdiction = raw_jurisdiction[:-1]
            # First check to see if jurisdiction exists, add otherwise
            geo_query = db.GqlQuery('SELECT __key__ FROM hierarchicalGeographic WHERE value=:1',
                                    raw_jurisdiction.lower())
            place_key = geo_query.get()
            if not place_key:
                new_geo = MODSData.hierarchicalGeographic(type="unknown",
                                                          value=raw_jurisdiction.lower(),
                                                          display_form=raw_jurisdiction)
                place_key = new_geo.put()
            corporate_creator = check_or_add_corporateBody(subfields_b[0])
            if corporate_creator.place.count(place_key) < 1:
                corporate_creator.place.append(place_key)
        elif first_indicator == '2':
            if len(subfields_b) < 1:
                corporate_creator = check_or_add_corporateBody(subfields_a[0])
            else: 
                parent_org = check_or_add_corporateBody(subfields_a[0])
                parent_org.put()
                if len(subfields_b) > 1:
                    corporate_creator = check_or_add_corporateBody(subfields_b[-1])
                    for b_field in subfields_b[:-1]:
                        new_note = MODSData.note(value=b_field)
                        new_note.put()
                        corporate_creator.notes.append(new_note.key())
                else:
                    corporate_creator = check_or_add_corporateBody(subfields_b[0])
                corporate_creator.associated_institution = parent_org
        corporate_creator.put()
        self.__check_add_author__(corporate_creator.key())


    def add_title(self,marc_field):
        """
        Method adds title from MARC 245 field

        Parameters:
        `marc_field`: Required, MARC 245 field.
        """
        non_sort = {'0':None,
                    '2':'A',
                    '3':'An',
                    '4':'The'}
        first_indicator,second_indicator = marc_field.indicators
        # Sets non-sort based on second indicator
        if non_sort.has_key(second_indicator):
            nonSort = non_sort.get(second_indicator)
        else:
            nonSort = None
        raw_title = marc_field.get_subfields('a')[0]
        if raw_title[-1] == r'/':
            raw_title = raw_title[:-1]
        raw_title = raw_title.strip()
        subTitle = marc_field.get_subfields('b')
        if len(subTitle) > 0:
            subTitle = subTitle[0]
        else:
            subTitle = None
        # Checks to see title exists in datastore
        query = db.GqlQuery("SELECT __key__ from titleInfo WHERE title=:1",
                            raw_title.lower())
        title = query.get()
        logging.error("Raw title is %s" % raw_title)
        if not title:
            title = MODSData.titleInfo(type="uniform",
                                       title=raw_title.lower(),
                                       display_form=raw_title)
            if subTitle:
                title.subTitle = subTitle
            if nonSort:
                title.nonSort = nonSort
            title.put()
        logging.error("Titles count=%s" % self.work.titles.count(title.key()))
        if self.work.titles.count(title.key()) < 1:
            self.work.titles.append(title.key())
        if self.expression.title != title:
            self.expression.title = title
        if self.manifestation.title != title:
            self.manifestation.title = title
        if self.endeavor.title != title:
            self.endeavor.title = title
        self.work.put()
        logging.error("Work title is %s" % self.work.get_title())
        self.expression.put()
        self.manifestation.put()
        self.endeavor.put()
       
    def add_edition(self,marc_field):
        """
        Method adds edition to manifestation
        """
        edition_statement = marc_field.get_subfields('a')[0]
        self.manifestation.edition = edition_statement
        self.manifestation.put()

    def add_publisher_info(self,marc_field):
        """
        Method checks/adds publisher as a CorporateBody to the
        manifestation. Adds publisher place to manifestation
        as well.

        Parameters:
        `marc_field`: Required, MARC 250 field 
        """
        all_places = marc_field.get_subfields('a')
        for place in all_places:
            self.add_publisher_place(place)

    def add_publisher_place(self,raw_place):
        """
        Method checks/adds publisher place to manifestation.

        Parameters:
        `raw_place`: Value from MARC 250 subfield a
        """
        corrected_place_re = re.compile(r'\[(w+)\]*')
        if corrected_place_re.search(raw_place):
            place_value = corrected_place_re.search(raw_place).groups()
            if re.search(r'S.l.',place_value):
                # Place is unknown, return from method
                return None
        else:
            place_value = raw_value
        
            
    def transform(self):
        r""" method transforms MARC record into eCataloger FRBR entities"""
        def check_record(entity_key):
            if not self.record.associated_entities.count(entity_key):
                self.record.associated_entities.append(entity_key)
        for record in self.marc_reader:
            self.process_leader(record.leader)
            if record['010']:
                self.add_lccn(record['010'])
            if record['020']:
                self.add_isbn(record.get_fields('020'))
            if record['022']:
                self.add_issn(record['022'])
            if record['035']:
                self.add_system_ctrl_num(record['035'])
            if record['040']:
                self.add_catalog(record['040'])
            if record['050']:
                self.add_lc_call_number(record['050'])
            if record['082']:
                self.add_dewey_call_number(record['082'])
            if record['100']:
                self.add_personal_name(record['100'])
            if record['245']:
                self.add_title(record['245'])
            if record['250']:
                self.add_edition(record['250'])
        work_key = self.work.put()
        check_record(work_key)
        if self.expression.work == None:
            self.expression.work = db.get(work_key)
        elif self.expression.work.key() != work_key:
            self.expression.work = db.get(work_key)
        expression_key = self.expression.put()
        check_record(expression_key)
        if not self.manifestation.expressions.count(expression_key):
            self.manifestation.expressions.append(expression_key)
        manifestation_key = self.manifestation.put()
        check_record(manifestation_key)
        
            
        
        
